package com.enterprise.application.spider;

import com.enterprise.application.di.spring.SpringLocator;
import com.enterprise.support.utility.Configuration;
import com.enterprise.support.utility.FileUtility;
import com.enterprise.support.utility.Validation;
import org.apache.log4j.PropertyConfigurator;
import org.dom4j.Document;
import org.dom4j.Element;
import org.dom4j.io.SAXReader;
import org.webharvest.definition.ScraperConfiguration;
import org.webharvest.runtime.Scraper;
import org.xml.sax.InputSource;

import java.io.*;
import java.util.*;

/**
 * Created by IntelliJ IDEA.
 * User: Administrator
 * Date: 2009-11-9
 * Time: 17:53:07
 * To change this template use File | Settings | File Templates.
 */
public class SpiderApplication {
    private static SpiderApplication application = new SpiderApplication();
    private String configLocation = Configuration.ConfHomeDirectory + "/classes/config/spider/spider-config.xml";
    private String workingdir = Configuration.ConfHomeDirectory + "/classes/config/spider/workingdir/";

    private Map<String, Scraper> scrapers = new HashMap<String, Scraper>();

    private Map<String, Object> utility = new HashMap<String, Object>();

    private Map<String, Object> logic = new HashMap<String, Object>();

    private boolean isDebug = false;

    private PrintWriter out = new PrintWriter(System.out);

    public void setOut(PrintWriter out) {
        this.out = out;
    }

    private SpiderApplication() {
        afterPropertiesSet();
        display();
    }
    //文章属性

    public static SpiderApplication getInstance() {
        return application;
    }

    private void debug(boolean isDebug) {
        if (!isDebug) return;
        Properties props = new Properties();
        props.setProperty("log4j.rootLogger", "INFO, stdout");
        props.setProperty("log4j.appender.stdout", "org.apache.log4j.ConsoleAppender");
        props.setProperty("log4j.appender.stdout.layout", "org.apache.log4j.PatternLayout");
        props.setProperty("log4j.appender.stdout.layout.ConversionPattern", "%-5p (%20F:%-3L) - %m\n");
        PropertyConfigurator.configure(props);
    }

    /**
     * 装载Resources
     *
     * @param configLocation resource文件Classpath路径
     */
    public Document load(String configLocation) {
        InputStream configIns = null;
        Document doc = null;
        try {
            configIns = new FileInputStream(configLocation);
            SAXReader reader = new SAXReader();
            doc = reader.read(configIns);
            configIns.close();
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            if (configIns != null) {
                try {
                    configIns.close();
                } catch (IOException ex) {
                }
            }
        }
        return doc;
    }

    private void initConfig(Document doc) {
        Element oRoot = doc.getRootElement();
        loadSpiderResource(oRoot);
    }

    private void loadSpiderResource(Element ele) {

        // add utility
        List<Element> utilitys = ele.elements("utility");
        if (utility != null) {
            for (Element temp : utilitys) {
                String name = temp.attributeValue("name", "");
                String className = temp.attributeValue("class", "");
                if (Validation.isEmpty(name) || Validation.isEmpty(className)) {
                    continue;
                }
                Object obj = parseClass(className);
                if (obj == null) continue;
                utility.put(name, obj);
            }
        }
        /*ServiceLocator*/
        utility.put("locator", SpringLocator.getApplicationContext());

        // add logic
        List<Element> logics = ele.elements("logic");
        if (utility != null) {
            for (Element temp : logics) {
                String name = temp.attributeValue("name", "");
                String className = temp.attributeValue("class", "");
                if (Validation.isEmpty(name) || Validation.isEmpty(className)) {
                    continue;
                }
                Object obj = null;
                try {
                    obj = SpringLocator.getBean(Class.forName(className));
                } catch (Exception ex) {
                    ex.printStackTrace();
                }
                if (obj == null) continue;
                logic.put(name, obj);
            }
        }

        /*Add Utility Var Context */
        ScraperFacadeImpl.getInstance().put("utility", utility);
        /*Add Logic Var Context */
        ScraperFacadeImpl.getInstance().put("logic", logic);

        //debug
        Element debugEle = ele.element("debug");
        if (debugEle != null) {
            String debugStr = debugEle.attributeValue("isDebug", "false");
            this.isDebug = "true".equalsIgnoreCase(debugStr);
            if (!Validation.isEmpty(debugEle.getTextTrim())) {
                this.workingdir = debugEle.getTextTrim();
            }
            //debug(this.isDebug);
        }
    }


    private Object parseClass(String className) {
        Object obj = null;
        try {
            obj = Class.forName(className).newInstance();
        } catch (Exception ex) {
            System.err.print(ex);
        }
        return obj;
    }

    public void afterPropertiesSet() {

        Document doc = load(configLocation);
        if (doc == null) {
            System.err.print("configLocation is error ");
            return;
        }
        initConfig(doc);
    }


    public Scraper getScraper(String name) {
        if (scrapers.containsKey(name)) {
            return scrapers.get(name);
        }
        return null;
    }

    public void setScrapers(Map<String, Scraper> scrapers) {
        this.scrapers = scrapers;
    }

    public void display() {
        out.println(" ##### 配置信息如下 ");
        out.println(" ##### 是否调试 " + (this.isDebug ? "是" : "否"));
        out.println(" ##### 调试文件路径 " + this.workingdir);
        out.println(" ##### 上下文工具如下 ");
        for (Iterator<String> it = utility.keySet().iterator(); it.hasNext(); ) {
            String key = it.next();
            Object value = utility.get(key);
            out.println(" ##### 工具  名称：" + key + " class：" + value);
        }
        out.println(" ##### Logic工具如下 ");
        for (Iterator<String> it = logic.keySet().iterator(); it.hasNext(); ) {
            String key = it.next();
            Object value = logic.get(key);
            out.println(" ##### 工具  名称：" + key + " class：" + value);
        }

    }


    /*重载*/
    public void execute(String filepath) {
        execute(filepath, new Callback() {
            public void execute(Scraper scraper) {
                scraper.execute();
                scraper.dispose();
            }
        });
    }

    /*开始一个爬虫*/
    public void execute(String content, Callback back) {
        long startTime = System.currentTimeMillis();
        Scraper scraper = scrapers.get(content);
        try {
            /*清理代理地址*/
            System.getProperties().remove("fetch.proxyHost");
            System.getProperties().remove("fetch.proxyPort");
            if (scrapers.get(content) == null) {
                InputStream stream = null;
                if (content.endsWith(".xml")) {
                    stream = new FileInputStream(new File(Configuration.ConfHomeDirectory + "/classes/config/spider/", content));
                } else {
                    StringBuffer string = new StringBuffer(content);
                    string.insert(string.indexOf("\">") + 3, "<include path=\"utility/functions.xml\"/>");
                    stream = new ByteArrayInputStream(string.toString().replace("_$zyf$_", "\\").replace("_$ljf$_", "&amp;").replace("_$dkh$_", "&gt;").replace("_$xkh$_", "&lt;").getBytes("utf-8"));
                }
                InputSource input = new InputSource(stream);
                input.setEncoding("utf-8");
                ScraperConfiguration config = new ScraperConfiguration(input);
                config.setSourceFile(new File(Configuration.ConfHomeDirectory + "/classes/config/spider/utility/"));
                scraper = new Scraper(config, Configuration.ConfHomeDirectory + "/classes/config/spider/utility/");
                scraper.setDebug(true);
                /*Add Utility Var Context */
                scraper.getContext().put("utility", utility);
                /*Add Logic Var Context */
                scraper.getContext().put("logic", logic);
                /*缓存*/
                // scrapers.put(content, scraper);
            }
            back.execute(scraper);
        } catch (Exception ex) {
            ex.printStackTrace();
            ex.printStackTrace(out);
            out.flush();
        } finally {
            if (scraper != null)
                scraper.dispose();
            out.println("\n总耗时: " + ((System.currentTimeMillis() - startTime) / 1000 / 60) + " 分种");
        }
    }

    public static void main(String[] args) throws FileNotFoundException {

        /*设置消息输出,流*/
        SpiderApplication spiderApplication = SpiderApplication.getInstance();
//        ((Map) ScraperFacadeImpl.getInstance().get("utility")).put("categorys", new CategoryImpl().findAllMapping());
//        ((Map) ScraperFacadeImpl.getInstance().get("utility")).put("origins", new OriginImpl().findAllMapping());
        ((Map) ScraperFacadeImpl.getInstance().get("utility")).put("out", new PrintWriter(System.out) {
            public void println() {
                super.println();
                super.flush();
            }
        });
        //startpage
        spiderApplication.execute(FileUtility.readStream(new FileInputStream(new File(Configuration.LibDirectory + "/classes/config/", "xml/article/chinaz/article.xml"))), new Callback() {
            //        spiderApplication.execute("xml/article/jb51/article.xml", new Callback() {
            public void execute(Scraper scraper) {
                //scraper.getContext().put("category", "328");
                scraper.getContext().setVar("startpage", "http://chinaz.com/Webbiz/BizPlan/");
                scraper.execute();
            }
        });

    }
}